{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "TODO Придумать как учитывать Kernel.calc_grad_mean для решения о делении ядра\n",
    "TODO Перенести код в библиотеку fractalnn.py\n",
    "TODO Согласовать код с библиотекой generate-nn/nbmodule.py\n",
    "TODO Заменить в Kernel.left, Kernel.right на Kernel.childs. Поменять соответствующие функции.\n",
    "TODO Упростить инициализацию Kernel: первичная - без параметров\n",
    "TODO Добавить в Kernel параметр kernel_size\n",
    "TODO Возможно хватит квадратной матрицы Kernel?\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn.functional as F\n",
    "from torch import nn\n",
    "from torch.utils.data import DataLoader\n",
    "from torchvision import datasets\n",
    "from torchvision.transforms import ToTensor\n",
    "from enum import Enum\n",
    "import copy\n",
    "import random\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[-0.4147, -0.0406],\n",
      "        [-0.4577,  0.0653],\n",
      "        [-0.1847, -0.2568],\n",
      "        [ 0.6002, -0.2063]], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([-0.6285,  0.0233,  0.0242,  0.6759], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "l = nn.Linear(2, 4)\n",
    "print(l.weight)\n",
    "print(l.bias)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([4])\n",
      "torch.Size([2])\n",
      "l.shape=(torch.Size([4, 2]), torch.Size([4]))\n",
      "l1.shape=(torch.Size([2, 2]), torch.Size([2]))\n",
      "l2.shape=(torch.Size([4, 2]), torch.Size([4]))\n",
      "tensor([[-0.4147, -0.0406],\n",
      "        [-0.4577,  0.0653],\n",
      "        [-0.1847, -0.2568],\n",
      "        [ 0.6002, -0.2063]], grad_fn=<MmBackward0>)\n",
      "tensor([-0.9493, -0.2121, -0.3761,  0.8543], grad_fn=<AddBackward0>)\n",
      "tensor([-0.9493, -0.2121, -0.3761,  0.8543], grad_fn=<AddBackward0>)\n"
     ]
    }
   ],
   "source": [
    "w2, w1 = torch.linalg.qr(l.weight, mode='reduced')\n",
    "b = l.bias\n",
    "b1 = torch.zeros(w1.shape[0])\n",
    "b2 = b\n",
    "print(b2.shape)\n",
    "print(b1.shape)\n",
    "print(f\"l.shape={(l.weight.shape, l.bias.shape)}\")\n",
    "\n",
    "l1 = nn.Linear(w1.shape[1], w1.shape[0])\n",
    "l1.weight.data = w1\n",
    "l1.bias.data = b1\n",
    "print(f\"l1.shape={(l1.weight.shape, l1.bias.shape)}\")\n",
    "\n",
    "l2 = nn.Linear(w2.shape[1], w2.shape[0])\n",
    "l2.weight.data = w2\n",
    "l2.bias.data = b2\n",
    "print(f\"l2.shape={(l2.weight.shape, l2.bias.shape)}\")\n",
    "\n",
    "print(l2.weight @ l1.weight)\n",
    "x = torch.randn(2,)\n",
    "print(l(x))\n",
    "print(l2(l1(x)))\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SplitType(Enum):\n",
    "    ADD = 1\n",
    "    MUL = 2\n",
    "\n",
    "def get_random_split_type():\n",
    "    return SplitType(random.randint(1, 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Kernel(\n",
      "  (kernel): Linear(in_features=2, out_features=4, bias=True)\n",
      "  (activation): SELU()\n",
      "  (seq): Sequential(\n",
      "    (0): Linear(in_features=2, out_features=4, bias=True)\n",
      "    (1): SELU()\n",
      "  )\n",
      "  (flatten): Flatten(start_dim=1, end_dim=-1)\n",
      ")\n",
      "tensor([[ 0.0533,  2.1987, -0.5621,  1.1301]], grad_fn=<EluBackward0>)\n",
      "tensor([[-0.1372,  2.0312, -0.5256,  1.0323]], grad_fn=<EluBackward0>)\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "\n",
    "\n",
    "class SplitType(Enum):\n",
    "    ADD = 1\n",
    "    MUL = 2\n",
    "\n",
    "def get_random_split_type():\n",
    "    return SplitType(random.randint(1, 2))\n",
    "\n",
    "class Kernel(nn.Module):\n",
    "    def __init__(self, weigth: torch.Tensor, bias: torch.Tensor, activation: nn.Module = nn.SELU(), device: str = \"cpu\"):\n",
    "        super(Kernel, self).__init__()\n",
    "        self.device = device\n",
    "        self.kernel = nn.Linear(weigth.shape[1], weigth.shape[0])\n",
    "        self.kernel.weight.data = weigth\n",
    "        self.kernel.bias.data = bias\n",
    "        self.activation = activation\n",
    "        self.seq = nn.Sequential(self.kernel, self.activation)\n",
    "        self.flatten = nn.Flatten()\n",
    "        self.left = None\n",
    "        self.right = None\n",
    "        self.split_mode = None\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.flatten(x)\n",
    "        # assert self.split_mode is None and self.left is None and self.right is None, f\"split_mode={self.split_mode} but left={self.left} and right={self.right}\"\n",
    "        if(self.is_leaf()):\n",
    "            return self.seq(x)\n",
    "        elif(self.split_mode == SplitType.ADD):\n",
    "            return (self.left(x) + self.right(x)) * 0.5\n",
    "        elif(self.split_mode == SplitType.MUL):\n",
    "            return self.right(self.left(x))\n",
    "        else:\n",
    "            raise ValueError(f\"Unknown split mode {self.split_mode}\")\n",
    "    \n",
    "    def calc_grad_mean(self):\n",
    "        '''Вычисляет среднее всех абсолютных величин градиентов параметров'''\n",
    "        mean = torch.tensor(0.0)\n",
    "        if(not self.is_leaf()): return mean\n",
    "        params = self.model.parameters(recurse=False)\n",
    "        with torch.no_grad():\n",
    "            n = 0\n",
    "            for p in params:\n",
    "                if p.grad is not None:\n",
    "                    n += 1\n",
    "                    mean += p.grad.abs().mean()\n",
    "            mean /= n\n",
    "        return mean\n",
    "    \n",
    "   \n",
    "    def is_leaf(self):\n",
    "        return self.left is None or self.right is None\n",
    "\n",
    "    def split_add(self):\n",
    "        # Для kernel = (right(x) + left(x)) * 0.5\n",
    "        assert self.is_leaf(), f\"split_add must perform only for leaf, but there are {self.left} and {self.right}\"\n",
    "        # device = next(self.kernel.parameters()).device\n",
    "        self.left = copy.deepcopy(self).to(self.device)\n",
    "        self.right = copy.deepcopy(self).to(self.device)\n",
    "        self.kernel = None\n",
    "        self.split_mode = SplitType.ADD        \n",
    "\n",
    "    def split_mul(self):\n",
    "        assert self.is_leaf(), f\"split_mul must perform only for leaf, but there are {self.left} and {self.right}\"\n",
    "        # Для kernel = right(left(x))\n",
    "        # W*x+b = W2*(W1*x+b1)+b2 = W2*W1*x+W2*b1+b2\n",
    "        # W2*W1 = W для поиска W2,W1 используем QR разложение\n",
    "        # Примем b1 = 0, тогда b2 = b\n",
    "        w = self.kernel.weight\n",
    "        b = self.kernel.bias\n",
    "        w2, w1 = torch.linalg.qr(w, mode='reduced')\n",
    "        b1 = torch.zeros(w1.shape[0])\n",
    "        b2 = b        \n",
    "        self.left = Kernel(w1, b1, device=self.device).to(self.device)\n",
    "        self.right = Kernel(w2, b2, device=self.device).to(self.device)\n",
    "        self.kernel = None\n",
    "        self.split_mode = SplitType.MUL\n",
    "        \n",
    "    \n",
    "    def split(self):\n",
    "        if(self.is_leaf()):\n",
    "            self.split_add()\n",
    "            self.left.split_mul()\n",
    "            self.right.split_mul()\n",
    "        else:\n",
    "            self.left.split()\n",
    "            self.right.split()\n",
    "    \n",
    "    def split_random_step(self):\n",
    "        if(self.is_leaf()):\n",
    "            split_type = get_random_split_type()\n",
    "            if(split_type == SplitType.ADD):\n",
    "                self.split_add()\n",
    "            elif(split_type == SplitType.MUL):\n",
    "                self.split_mul()\n",
    "        else:\n",
    "            if(random.random()<0.5):\n",
    "                self.left.split_random_step()\n",
    "            else:\n",
    "                self.right.split_random_step()\n",
    "            \n",
    "\n",
    "in_features = 2\n",
    "out_features = 4\n",
    "# Генерируем случайные веса и смещения\n",
    "w = torch.randn(out_features, in_features)\n",
    "b = torch.randn(out_features)\n",
    "# Создаем модель\n",
    "model = Kernel(w, b, device=\"cpu\")\n",
    "print(model)\n",
    "# Проверяем, что она работает\n",
    "x = torch.randn(1, in_features)\n",
    "y = model(x)\n",
    "\n",
    "# Разделяем его\n",
    "model.split_mul()\n",
    "# Проверяем, что она работает\n",
    "print(model(x))\n",
    "print(y)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of X [N, C, H, W]: torch.Size([10000, 1, 28, 28])\n",
      "Shape of y: torch.Size([10000]) torch.int64\n"
     ]
    }
   ],
   "source": [
    "# Download training data from open datasets.\n",
    "training_data = datasets.FashionMNIST(\n",
    "    root=\"data\",\n",
    "    train=True,\n",
    "    download=True,\n",
    "    transform=ToTensor(),\n",
    ")\n",
    "\n",
    "# Download test data from open datasets.\n",
    "test_data = datasets.FashionMNIST(\n",
    "    root=\"data\",\n",
    "    train=False,\n",
    "    download=True,\n",
    "    transform=ToTensor(),\n",
    ")\n",
    "batch_size = 2**14\n",
    "\n",
    "train_dataloader = DataLoader(training_data, batch_size=batch_size)\n",
    "test_dataloader = DataLoader(test_data, batch_size=batch_size)\n",
    "\n",
    "for X, y in test_dataloader:\n",
    "    print(f\"Shape of X [N, C, H, W]: {X.shape}\")\n",
    "    print(f\"Shape of y: {y.shape} {y.dtype}\")\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using cuda:0 device\n"
     ]
    }
   ],
   "source": [
    "# Get cpu, gpu or mps device for training.\n",
    "# device = \"cuda\"\n",
    "# (\n",
    "#     \"cuda\"\n",
    "#     if torch.cuda.is_available()\n",
    "#     else \"mps\"\n",
    "#     if torch.backends.mps.is_available()\n",
    "#     else \"cpu\"\n",
    "# )\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "print(f\"Using {device} device\")\n",
    "\n",
    "in_features = 28*28\n",
    "out_features = 10\n",
    "# w = torch.randn(out_features, in_features)\n",
    "# b = torch.randn(out_features)\n",
    "# # Создаем модель\n",
    "\n",
    "# model = Kernel(w, b).to(device)\n",
    "# print(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Класс обертка над Kernel для возможности управления делением ядер\n",
    "class SupervisorModel(nn.Module):\n",
    "    def __init__(self, in_features: int, out_features: int, kernel_size: int = 8, device: str = \"cpu\"):\n",
    "        super().__init__()\n",
    "        self.device = device\n",
    "        self.in_features = in_features\n",
    "        self.out_features = out_features\n",
    "        self.kernel_size = kernel_size\n",
    "        self.flatten = nn.Flatten()\n",
    "        w = torch.randn(self.kernel_size, self.kernel_size)\n",
    "        b = torch.randn(self.kernel_size)\n",
    "        self.kernel = Kernel(w, b, device=self.device)\n",
    "        self.model = nn.Sequential(            \n",
    "            nn.Linear(in_features, kernel_size),\n",
    "            self.kernel,\n",
    "            nn.Linear(kernel_size, out_features)\n",
    "            )\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.flatten(x)\n",
    "        return self.model(x)\n",
    "\n",
    "    def split_random(self):\n",
    "        self.kernel.split_random_step()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SupervisorModel(\n",
      "  (flatten): Flatten(start_dim=1, end_dim=-1)\n",
      "  (kernel): Kernel(\n",
      "    (kernel): Linear(in_features=64, out_features=64, bias=True)\n",
      "    (activation): SELU()\n",
      "    (seq): Sequential(\n",
      "      (0): Linear(in_features=64, out_features=64, bias=True)\n",
      "      (1): SELU()\n",
      "    )\n",
      "    (flatten): Flatten(start_dim=1, end_dim=-1)\n",
      "  )\n",
      "  (model): Sequential(\n",
      "    (0): Linear(in_features=784, out_features=64, bias=True)\n",
      "    (1): Kernel(\n",
      "      (kernel): Linear(in_features=64, out_features=64, bias=True)\n",
      "      (activation): SELU()\n",
      "      (seq): Sequential(\n",
      "        (0): Linear(in_features=64, out_features=64, bias=True)\n",
      "        (1): SELU()\n",
      "      )\n",
      "      (flatten): Flatten(start_dim=1, end_dim=-1)\n",
      "    )\n",
      "    (2): Linear(in_features=64, out_features=10, bias=True)\n",
      "  )\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "model = SupervisorModel(in_features, out_features, 64, device).to(device)\n",
    "print(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(dataloader, model, loss_fn, optimizer):\n",
    "    size = len(dataloader.dataset)\n",
    "    model.train()\n",
    "    for batch, (X, y) in enumerate(dataloader):\n",
    "        X, y = X.to(device), y.to(device)\n",
    "\n",
    "        # Compute prediction error\n",
    "        pred = model(X)\n",
    "        loss = loss_fn(pred, y)\n",
    "\n",
    "        # Backpropagation\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        if batch % 2 == 0:\n",
    "            loss, current = loss.item(), (batch + 1) * len(X)\n",
    "            print(f\"loss: {loss:>7f}  [{current:>5d}/{size:>5d}]\")\n",
    "\n",
    "def test(dataloader, model, loss_fn):\n",
    "    size = len(dataloader.dataset)\n",
    "    num_batches = len(dataloader)\n",
    "    model.eval()\n",
    "    test_loss, correct = 0, 0\n",
    "    with torch.no_grad():\n",
    "        for X, y in dataloader:\n",
    "            X, y = X.to(device), y.to(device)\n",
    "            pred = model(X)\n",
    "            test_loss += loss_fn(pred, y).item()\n",
    "            correct += (pred.argmax(1) == y).type(torch.float).sum().item()\n",
    "    test_loss /= num_batches\n",
    "    correct /= size\n",
    "    print(f\"Test Error: \\n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evolution 1\n",
      "*********************************************\n",
      "Params=55050\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 2.793095  [16384/60000]\n",
      "loss: 2.558595  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 10.5%, Avg loss: 2.434660 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 2.429502  [16384/60000]\n",
      "loss: 2.342494  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 14.7%, Avg loss: 2.279345 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 2.274088  [16384/60000]\n",
      "loss: 2.214160  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 19.3%, Avg loss: 2.165634 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 2.160770  [16384/60000]\n",
      "loss: 2.109282  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 24.7%, Avg loss: 2.068479 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 2.063811  [16384/60000]\n",
      "loss: 2.017225  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 30.3%, Avg loss: 1.982132 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 1.977349  [16384/60000]\n",
      "loss: 1.934452  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 35.7%, Avg loss: 1.904032 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 1.899011  [16384/60000]\n",
      "loss: 1.859276  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 40.6%, Avg loss: 1.832885 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 1.827507  [16384/60000]\n",
      "loss: 1.790690  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 44.4%, Avg loss: 1.767995 \n",
      "\n",
      "Evolution 2\n",
      "*********************************************\n",
      "Params=63370\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 1.814697  [16384/60000]\n",
      "loss: 1.775732  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 47.4%, Avg loss: 1.750333 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 1.744813  [16384/60000]\n",
      "loss: 1.712330  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 51.1%, Avg loss: 1.693390 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 1.687552  [16384/60000]\n",
      "loss: 1.657478  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 53.8%, Avg loss: 1.642163 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 1.635796  [16384/60000]\n",
      "loss: 1.607919  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 56.1%, Avg loss: 1.595407 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 1.588450  [16384/60000]\n",
      "loss: 1.562715  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 57.8%, Avg loss: 1.552647 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 1.545122  [16384/60000]\n",
      "loss: 1.521379  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 59.2%, Avg loss: 1.513549 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 1.505470  [16384/60000]\n",
      "loss: 1.483565  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 60.3%, Avg loss: 1.477803 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 1.469208  [16384/60000]\n",
      "loss: 1.448959  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 61.1%, Avg loss: 1.445018 \n",
      "\n",
      "Evolution 3\n",
      "*********************************************\n",
      "Params=71690\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 1.422073  [16384/60000]\n",
      "loss: 1.402206  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 62.0%, Avg loss: 1.398837 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 1.388608  [16384/60000]\n",
      "loss: 1.370360  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 62.6%, Avg loss: 1.368923 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 1.358362  [16384/60000]\n",
      "loss: 1.341393  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 63.2%, Avg loss: 1.341642 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 1.330724  [16384/60000]\n",
      "loss: 1.314779  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 63.7%, Avg loss: 1.316495 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 1.305196  [16384/60000]\n",
      "loss: 1.290145  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 64.2%, Avg loss: 1.293181 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 1.281517  [16384/60000]\n",
      "loss: 1.267292  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 64.8%, Avg loss: 1.271470 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 1.259458  [16384/60000]\n",
      "loss: 1.245991  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 65.2%, Avg loss: 1.251171 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 1.238792  [16384/60000]\n",
      "loss: 1.226034  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 65.6%, Avg loss: 1.232118 \n",
      "\n",
      "Evolution 4\n",
      "*********************************************\n",
      "Params=80010\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 1.198087  [16384/60000]\n",
      "loss: 1.185680  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 66.1%, Avg loss: 1.192968 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 1.179398  [16384/60000]\n",
      "loss: 1.167652  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 66.5%, Avg loss: 1.175794 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 1.161913  [16384/60000]\n",
      "loss: 1.150710  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 66.8%, Avg loss: 1.159685 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 1.145479  [16384/60000]\n",
      "loss: 1.134778  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 67.1%, Avg loss: 1.144469 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 1.129978  [16384/60000]\n",
      "loss: 1.119753  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 67.4%, Avg loss: 1.130033 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 1.115334  [16384/60000]\n",
      "loss: 1.105523  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 67.7%, Avg loss: 1.116300 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 1.101443  [16384/60000]\n",
      "loss: 1.091977  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 67.9%, Avg loss: 1.103251 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 1.088264  [16384/60000]\n",
      "loss: 1.079113  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 68.2%, Avg loss: 1.090863 \n",
      "\n",
      "Evolution 5\n",
      "*********************************************\n",
      "Params=88330\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 1.056031  [16384/60000]\n",
      "loss: 1.046940  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 68.5%, Avg loss: 1.059732 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 1.044087  [16384/60000]\n",
      "loss: 1.035278  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 68.7%, Avg loss: 1.048540 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 1.032747  [16384/60000]\n",
      "loss: 1.024203  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 68.9%, Avg loss: 1.037894 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 1.021904  [16384/60000]\n",
      "loss: 1.013649  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 69.1%, Avg loss: 1.027814 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 1.011613  [16384/60000]\n",
      "loss: 1.003623  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 69.4%, Avg loss: 1.018229 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 1.001841  [16384/60000]\n",
      "loss: 0.994064  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 69.5%, Avg loss: 1.009064 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.992500  [16384/60000]\n",
      "loss: 0.984913  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 69.7%, Avg loss: 1.000283 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.983564  [16384/60000]\n",
      "loss: 0.976134  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 69.9%, Avg loss: 0.991892 \n",
      "\n",
      "Evolution 6\n",
      "*********************************************\n",
      "Params=100810\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.974981  [16384/60000]\n",
      "loss: 0.967719  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 70.1%, Avg loss: 0.983835 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.966690  [16384/60000]\n",
      "loss: 0.959620  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 70.3%, Avg loss: 0.976105 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.958678  [16384/60000]\n",
      "loss: 0.951842  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 70.4%, Avg loss: 0.968663 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.950932  [16384/60000]\n",
      "loss: 0.944345  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 70.5%, Avg loss: 0.961464 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.943439  [16384/60000]\n",
      "loss: 0.937138  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 70.6%, Avg loss: 0.954493 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.936242  [16384/60000]\n",
      "loss: 0.930180  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 70.8%, Avg loss: 0.947788 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.929280  [16384/60000]\n",
      "loss: 0.923466  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 70.9%, Avg loss: 0.941285 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.922559  [16384/60000]\n",
      "loss: 0.916959  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 71.0%, Avg loss: 0.934965 \n",
      "\n",
      "Evolution 7\n",
      "*********************************************\n",
      "Params=113290\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.916054  [16384/60000]\n",
      "loss: 0.910651  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 71.2%, Avg loss: 0.928845 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.909758  [16384/60000]\n",
      "loss: 0.904564  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 71.3%, Avg loss: 0.922908 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.903679  [16384/60000]\n",
      "loss: 0.898655  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 71.5%, Avg loss: 0.917104 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.897776  [16384/60000]\n",
      "loss: 0.892927  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 71.5%, Avg loss: 0.911496 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.892068  [16384/60000]\n",
      "loss: 0.887358  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 71.8%, Avg loss: 0.906074 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.886522  [16384/60000]\n",
      "loss: 0.881959  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 72.0%, Avg loss: 0.900806 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.881139  [16384/60000]\n",
      "loss: 0.876696  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 72.1%, Avg loss: 0.895679 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.875889  [16384/60000]\n",
      "loss: 0.871582  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 72.2%, Avg loss: 0.890690 \n",
      "\n",
      "Evolution 8\n",
      "*********************************************\n",
      "Params=121610\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.862952  [16384/60000]\n",
      "loss: 0.858865  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 72.5%, Avg loss: 0.878528 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.858043  [16384/60000]\n",
      "loss: 0.854111  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 72.6%, Avg loss: 0.873891 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.853249  [16384/60000]\n",
      "loss: 0.849497  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 72.7%, Avg loss: 0.869369 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.848576  [16384/60000]\n",
      "loss: 0.844984  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 72.9%, Avg loss: 0.864978 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.844039  [16384/60000]\n",
      "loss: 0.840589  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 73.0%, Avg loss: 0.860661 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.839640  [16384/60000]\n",
      "loss: 0.836277  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 73.2%, Avg loss: 0.856454 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.835367  [16384/60000]\n",
      "loss: 0.832059  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 73.4%, Avg loss: 0.852358 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.831185  [16384/60000]\n",
      "loss: 0.827953  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 73.5%, Avg loss: 0.848340 \n",
      "\n",
      "Evolution 9\n",
      "*********************************************\n",
      "Params=125770\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.820157  [16384/60000]\n",
      "loss: 0.817098  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 73.6%, Avg loss: 0.837983 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.816229  [16384/60000]\n",
      "loss: 0.813292  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 73.8%, Avg loss: 0.834223 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.812404  [16384/60000]\n",
      "loss: 0.809572  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 73.9%, Avg loss: 0.830523 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.808687  [16384/60000]\n",
      "loss: 0.805947  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 74.1%, Avg loss: 0.826902 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.805075  [16384/60000]\n",
      "loss: 0.802398  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 74.2%, Avg loss: 0.823371 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.801532  [16384/60000]\n",
      "loss: 0.798919  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 74.2%, Avg loss: 0.819911 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.798052  [16384/60000]\n",
      "loss: 0.795515  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 74.3%, Avg loss: 0.816555 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.794666  [16384/60000]\n",
      "loss: 0.792153  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 74.4%, Avg loss: 0.813283 \n",
      "\n",
      "Evolution 10\n",
      "*********************************************\n",
      "Params=142410\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.791352  [16384/60000]\n",
      "loss: 0.788873  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 74.5%, Avg loss: 0.810065 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.788109  [16384/60000]\n",
      "loss: 0.785657  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 74.6%, Avg loss: 0.806915 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.784930  [16384/60000]\n",
      "loss: 0.782532  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 74.6%, Avg loss: 0.803856 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.781854  [16384/60000]\n",
      "loss: 0.779449  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 74.7%, Avg loss: 0.800825 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.778834  [16384/60000]\n",
      "loss: 0.776429  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 74.7%, Avg loss: 0.797857 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.775857  [16384/60000]\n",
      "loss: 0.773481  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 74.8%, Avg loss: 0.794961 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.772928  [16384/60000]\n",
      "loss: 0.770591  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 74.8%, Avg loss: 0.792104 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.770046  [16384/60000]\n",
      "loss: 0.767749  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 74.9%, Avg loss: 0.789337 \n",
      "\n",
      "Evolution 11\n",
      "*********************************************\n",
      "Params=159050\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.767243  [16384/60000]\n",
      "loss: 0.764960  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 74.9%, Avg loss: 0.786626 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.764476  [16384/60000]\n",
      "loss: 0.762220  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 75.0%, Avg loss: 0.783974 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.761745  [16384/60000]\n",
      "loss: 0.759545  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 75.1%, Avg loss: 0.781344 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.759090  [16384/60000]\n",
      "loss: 0.756891  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 75.2%, Avg loss: 0.778760 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.756484  [16384/60000]\n",
      "loss: 0.754301  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 75.2%, Avg loss: 0.776241 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.753909  [16384/60000]\n",
      "loss: 0.751766  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 75.3%, Avg loss: 0.773742 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.751371  [16384/60000]\n",
      "loss: 0.749278  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 75.3%, Avg loss: 0.771320 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.748860  [16384/60000]\n",
      "loss: 0.746809  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 75.4%, Avg loss: 0.768911 \n",
      "\n",
      "Evolution 12\n",
      "*********************************************\n",
      "Params=171530\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.746401  [16384/60000]\n",
      "loss: 0.744384  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 75.4%, Avg loss: 0.766532 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.743968  [16384/60000]\n",
      "loss: 0.742017  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 75.4%, Avg loss: 0.764207 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.741581  [16384/60000]\n",
      "loss: 0.739653  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 75.4%, Avg loss: 0.761904 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.739255  [16384/60000]\n",
      "loss: 0.737350  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 75.5%, Avg loss: 0.759655 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.736959  [16384/60000]\n",
      "loss: 0.735084  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 75.6%, Avg loss: 0.757423 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.734695  [16384/60000]\n",
      "loss: 0.732827  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 75.7%, Avg loss: 0.755235 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.732484  [16384/60000]\n",
      "loss: 0.730633  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 75.7%, Avg loss: 0.753088 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.730297  [16384/60000]\n",
      "loss: 0.728469  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 75.8%, Avg loss: 0.750979 \n",
      "\n",
      "Evolution 13\n",
      "*********************************************\n",
      "Params=171530\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.725445  [16384/60000]\n",
      "loss: 0.723676  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 75.8%, Avg loss: 0.746409 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.723374  [16384/60000]\n",
      "loss: 0.721609  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 75.9%, Avg loss: 0.744396 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.721315  [16384/60000]\n",
      "loss: 0.719579  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 75.9%, Avg loss: 0.742404 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.719296  [16384/60000]\n",
      "loss: 0.717593  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 75.9%, Avg loss: 0.740477 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.717307  [16384/60000]\n",
      "loss: 0.715625  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 76.0%, Avg loss: 0.738540 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.715336  [16384/60000]\n",
      "loss: 0.713696  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 76.0%, Avg loss: 0.736655 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.713371  [16384/60000]\n",
      "loss: 0.711776  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 76.1%, Avg loss: 0.734795 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.711471  [16384/60000]\n",
      "loss: 0.709897  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 76.1%, Avg loss: 0.732971 \n",
      "\n",
      "Evolution 14\n",
      "*********************************************\n",
      "Params=175690\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.708311  [16384/60000]\n",
      "loss: 0.706765  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 76.1%, Avg loss: 0.729989 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.706458  [16384/60000]\n",
      "loss: 0.704920  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 76.2%, Avg loss: 0.728207 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.704625  [16384/60000]\n",
      "loss: 0.703073  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 76.2%, Avg loss: 0.726484 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.702829  [16384/60000]\n",
      "loss: 0.701310  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 76.3%, Avg loss: 0.724750 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.701061  [16384/60000]\n",
      "loss: 0.699548  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 76.3%, Avg loss: 0.723059 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.699310  [16384/60000]\n",
      "loss: 0.697841  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 76.3%, Avg loss: 0.721396 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.697581  [16384/60000]\n",
      "loss: 0.696169  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 76.4%, Avg loss: 0.719748 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.695869  [16384/60000]\n",
      "loss: 0.694506  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 76.4%, Avg loss: 0.718103 \n",
      "\n",
      "Evolution 15\n",
      "*********************************************\n",
      "Params=175690\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.692965  [16384/60000]\n",
      "loss: 0.691675  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 76.4%, Avg loss: 0.715378 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.691336  [16384/60000]\n",
      "loss: 0.690058  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 76.6%, Avg loss: 0.713811 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.689722  [16384/60000]\n",
      "loss: 0.688449  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 76.6%, Avg loss: 0.712263 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.688146  [16384/60000]\n",
      "loss: 0.686883  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 76.6%, Avg loss: 0.710728 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.686566  [16384/60000]\n",
      "loss: 0.685345  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 76.7%, Avg loss: 0.709197 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.685011  [16384/60000]\n",
      "loss: 0.683820  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 76.7%, Avg loss: 0.707689 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.683473  [16384/60000]\n",
      "loss: 0.682294  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 76.7%, Avg loss: 0.706193 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.681972  [16384/60000]\n",
      "loss: 0.680802  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 76.8%, Avg loss: 0.704750 \n",
      "\n",
      "Evolution 16\n",
      "*********************************************\n",
      "Params=179850\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.678178  [16384/60000]\n",
      "loss: 0.677061  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 76.7%, Avg loss: 0.701240 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.676733  [16384/60000]\n",
      "loss: 0.675632  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 76.8%, Avg loss: 0.699842 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.675304  [16384/60000]\n",
      "loss: 0.674200  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 76.9%, Avg loss: 0.698435 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.673879  [16384/60000]\n",
      "loss: 0.672818  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 76.9%, Avg loss: 0.697064 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.672454  [16384/60000]\n",
      "loss: 0.671422  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 77.0%, Avg loss: 0.695681 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.671070  [16384/60000]\n",
      "loss: 0.670043  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 77.0%, Avg loss: 0.694361 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.669690  [16384/60000]\n",
      "loss: 0.668693  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 77.1%, Avg loss: 0.693030 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.668345  [16384/60000]\n",
      "loss: 0.667322  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 77.1%, Avg loss: 0.691728 \n",
      "\n",
      "Evolution 17\n",
      "*********************************************\n",
      "Params=192330\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.667003  [16384/60000]\n",
      "loss: 0.665986  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 77.1%, Avg loss: 0.690449 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.665662  [16384/60000]\n",
      "loss: 0.664665  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 77.2%, Avg loss: 0.689228 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.664325  [16384/60000]\n",
      "loss: 0.663370  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 77.2%, Avg loss: 0.687968 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.663015  [16384/60000]\n",
      "loss: 0.662104  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 77.2%, Avg loss: 0.686731 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.661718  [16384/60000]\n",
      "loss: 0.660860  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 77.2%, Avg loss: 0.685543 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.660456  [16384/60000]\n",
      "loss: 0.659622  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 77.2%, Avg loss: 0.684349 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.659180  [16384/60000]\n",
      "loss: 0.658406  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 77.2%, Avg loss: 0.683140 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.657951  [16384/60000]\n",
      "loss: 0.657205  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 77.3%, Avg loss: 0.681970 \n",
      "\n",
      "Evolution 18\n",
      "*********************************************\n",
      "Params=204810\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.656734  [16384/60000]\n",
      "loss: 0.655986  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 77.3%, Avg loss: 0.680797 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.655540  [16384/60000]\n",
      "loss: 0.654778  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 77.3%, Avg loss: 0.679647 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.654350  [16384/60000]\n",
      "loss: 0.653591  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 77.4%, Avg loss: 0.678466 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.653173  [16384/60000]\n",
      "loss: 0.652425  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 77.4%, Avg loss: 0.677330 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.652006  [16384/60000]\n",
      "loss: 0.651284  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 77.4%, Avg loss: 0.676192 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.650852  [16384/60000]\n",
      "loss: 0.650135  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 77.5%, Avg loss: 0.675041 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.649704  [16384/60000]\n",
      "loss: 0.649010  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 77.5%, Avg loss: 0.673923 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.648576  [16384/60000]\n",
      "loss: 0.647891  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 77.6%, Avg loss: 0.672815 \n",
      "\n",
      "Evolution 19\n",
      "*********************************************\n",
      "Params=217290\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.647463  [16384/60000]\n",
      "loss: 0.646770  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 77.6%, Avg loss: 0.671723 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.646344  [16384/60000]\n",
      "loss: 0.645672  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 77.6%, Avg loss: 0.670668 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.645247  [16384/60000]\n",
      "loss: 0.644619  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 77.7%, Avg loss: 0.669599 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.644160  [16384/60000]\n",
      "loss: 0.643576  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 77.7%, Avg loss: 0.668531 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.643089  [16384/60000]\n",
      "loss: 0.642515  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 77.7%, Avg loss: 0.667448 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.641988  [16384/60000]\n",
      "loss: 0.641491  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 77.7%, Avg loss: 0.666391 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.640889  [16384/60000]\n",
      "loss: 0.640458  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 77.8%, Avg loss: 0.665363 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.639838  [16384/60000]\n",
      "loss: 0.639447  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 77.8%, Avg loss: 0.664363 \n",
      "\n",
      "Evolution 20\n",
      "*********************************************\n",
      "Params=225610\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.634907  [16384/60000]\n",
      "loss: 0.634685  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 77.7%, Avg loss: 0.659892 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.633826  [16384/60000]\n",
      "loss: 0.633701  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 77.8%, Avg loss: 0.658905 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.632789  [16384/60000]\n",
      "loss: 0.632719  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 77.9%, Avg loss: 0.657947 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.631792  [16384/60000]\n",
      "loss: 0.631725  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 77.9%, Avg loss: 0.657019 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.630800  [16384/60000]\n",
      "loss: 0.630759  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.0%, Avg loss: 0.656086 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.629830  [16384/60000]\n",
      "loss: 0.629805  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.0%, Avg loss: 0.655157 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.628863  [16384/60000]\n",
      "loss: 0.628825  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.0%, Avg loss: 0.654258 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.627892  [16384/60000]\n",
      "loss: 0.627882  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.1%, Avg loss: 0.653343 \n",
      "\n",
      "Evolution 21\n",
      "*********************************************\n",
      "Params=242250\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.626942  [16384/60000]\n",
      "loss: 0.626971  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.0%, Avg loss: 0.652395 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.625979  [16384/60000]\n",
      "loss: 0.626003  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.1%, Avg loss: 0.651478 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.625026  [16384/60000]\n",
      "loss: 0.625065  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.1%, Avg loss: 0.650568 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.624095  [16384/60000]\n",
      "loss: 0.624155  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.1%, Avg loss: 0.649699 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.623199  [16384/60000]\n",
      "loss: 0.623267  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.2%, Avg loss: 0.648815 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.622280  [16384/60000]\n",
      "loss: 0.622363  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.2%, Avg loss: 0.647934 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.621354  [16384/60000]\n",
      "loss: 0.621473  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.2%, Avg loss: 0.647017 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.620449  [16384/60000]\n",
      "loss: 0.620587  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.2%, Avg loss: 0.646164 \n",
      "\n",
      "Evolution 22\n",
      "*********************************************\n",
      "Params=250570\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.617791  [16384/60000]\n",
      "loss: 0.617966  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.2%, Avg loss: 0.643802 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.616912  [16384/60000]\n",
      "loss: 0.617112  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.2%, Avg loss: 0.642947 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.616041  [16384/60000]\n",
      "loss: 0.616254  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.2%, Avg loss: 0.642086 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.615175  [16384/60000]\n",
      "loss: 0.615398  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.3%, Avg loss: 0.641261 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.614348  [16384/60000]\n",
      "loss: 0.614539  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.3%, Avg loss: 0.640426 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.613514  [16384/60000]\n",
      "loss: 0.613662  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.4%, Avg loss: 0.639602 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.612669  [16384/60000]\n",
      "loss: 0.612805  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.4%, Avg loss: 0.638823 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.611816  [16384/60000]\n",
      "loss: 0.611963  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.4%, Avg loss: 0.638028 \n",
      "\n",
      "Evolution 23\n",
      "*********************************************\n",
      "Params=263050\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.610973  [16384/60000]\n",
      "loss: 0.611130  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.4%, Avg loss: 0.637250 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.610175  [16384/60000]\n",
      "loss: 0.610270  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.4%, Avg loss: 0.636486 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.609375  [16384/60000]\n",
      "loss: 0.609479  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.4%, Avg loss: 0.635677 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.608521  [16384/60000]\n",
      "loss: 0.608675  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.5%, Avg loss: 0.634961 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.607750  [16384/60000]\n",
      "loss: 0.607888  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.5%, Avg loss: 0.634179 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.606925  [16384/60000]\n",
      "loss: 0.607122  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.5%, Avg loss: 0.633416 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.606135  [16384/60000]\n",
      "loss: 0.606337  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.6%, Avg loss: 0.632658 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.605310  [16384/60000]\n",
      "loss: 0.605597  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.6%, Avg loss: 0.631928 \n",
      "\n",
      "Evolution 24\n",
      "*********************************************\n",
      "Params=279690\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.604542  [16384/60000]\n",
      "loss: 0.604838  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.6%, Avg loss: 0.631200 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.603791  [16384/60000]\n",
      "loss: 0.604077  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.6%, Avg loss: 0.630453 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.603025  [16384/60000]\n",
      "loss: 0.603352  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.6%, Avg loss: 0.629728 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.602257  [16384/60000]\n",
      "loss: 0.602607  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.6%, Avg loss: 0.628958 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.601489  [16384/60000]\n",
      "loss: 0.601891  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.6%, Avg loss: 0.628291 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.600733  [16384/60000]\n",
      "loss: 0.601137  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.7%, Avg loss: 0.627581 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.599964  [16384/60000]\n",
      "loss: 0.600425  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.7%, Avg loss: 0.626859 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.599220  [16384/60000]\n",
      "loss: 0.599692  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.7%, Avg loss: 0.626163 \n",
      "\n",
      "Evolution 25\n",
      "*********************************************\n",
      "Params=292170\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.598525  [16384/60000]\n",
      "loss: 0.598998  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.7%, Avg loss: 0.625471 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.597789  [16384/60000]\n",
      "loss: 0.598325  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.7%, Avg loss: 0.624802 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.597080  [16384/60000]\n",
      "loss: 0.597598  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.7%, Avg loss: 0.624076 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.596389  [16384/60000]\n",
      "loss: 0.596898  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.8%, Avg loss: 0.623423 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.595683  [16384/60000]\n",
      "loss: 0.596216  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.8%, Avg loss: 0.622761 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.595012  [16384/60000]\n",
      "loss: 0.595532  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.8%, Avg loss: 0.622026 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.594298  [16384/60000]\n",
      "loss: 0.594839  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.8%, Avg loss: 0.621355 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.593606  [16384/60000]\n",
      "loss: 0.594144  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.9%, Avg loss: 0.620732 \n",
      "\n",
      "Evolution 26\n",
      "*********************************************\n",
      "Params=308810\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.592943  [16384/60000]\n",
      "loss: 0.593484  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.8%, Avg loss: 0.620045 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.592267  [16384/60000]\n",
      "loss: 0.592826  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.9%, Avg loss: 0.619421 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.591610  [16384/60000]\n",
      "loss: 0.592170  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.9%, Avg loss: 0.618755 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.590951  [16384/60000]\n",
      "loss: 0.591493  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 78.9%, Avg loss: 0.618137 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.590298  [16384/60000]\n",
      "loss: 0.590848  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.0%, Avg loss: 0.617532 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.589619  [16384/60000]\n",
      "loss: 0.590203  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.0%, Avg loss: 0.616913 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.589006  [16384/60000]\n",
      "loss: 0.589540  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.0%, Avg loss: 0.616315 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.588361  [16384/60000]\n",
      "loss: 0.588872  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.0%, Avg loss: 0.615762 \n",
      "\n",
      "Evolution 27\n",
      "*********************************************\n",
      "Params=321290\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.587735  [16384/60000]\n",
      "loss: 0.588265  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.0%, Avg loss: 0.615126 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.587096  [16384/60000]\n",
      "loss: 0.587655  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.0%, Avg loss: 0.614567 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.586476  [16384/60000]\n",
      "loss: 0.587024  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.1%, Avg loss: 0.613921 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.585822  [16384/60000]\n",
      "loss: 0.586407  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.1%, Avg loss: 0.613361 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.585226  [16384/60000]\n",
      "loss: 0.585823  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.1%, Avg loss: 0.612790 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.584593  [16384/60000]\n",
      "loss: 0.585203  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.1%, Avg loss: 0.612212 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.584001  [16384/60000]\n",
      "loss: 0.584600  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.1%, Avg loss: 0.611589 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.583391  [16384/60000]\n",
      "loss: 0.584007  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.1%, Avg loss: 0.611080 \n",
      "\n",
      "Evolution 28\n",
      "*********************************************\n",
      "Params=329610\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.581254  [16384/60000]\n",
      "loss: 0.581962  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.1%, Avg loss: 0.609151 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.580649  [16384/60000]\n",
      "loss: 0.581347  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.1%, Avg loss: 0.608598 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.580069  [16384/60000]\n",
      "loss: 0.580767  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.2%, Avg loss: 0.608051 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.579468  [16384/60000]\n",
      "loss: 0.580210  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.2%, Avg loss: 0.607498 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.578886  [16384/60000]\n",
      "loss: 0.579628  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.2%, Avg loss: 0.606963 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.578309  [16384/60000]\n",
      "loss: 0.579100  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.2%, Avg loss: 0.606426 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.577726  [16384/60000]\n",
      "loss: 0.578516  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.3%, Avg loss: 0.605888 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.577151  [16384/60000]\n",
      "loss: 0.577956  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.3%, Avg loss: 0.605351 \n",
      "\n",
      "Evolution 29\n",
      "*********************************************\n",
      "Params=342090\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.576584  [16384/60000]\n",
      "loss: 0.577404  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.3%, Avg loss: 0.604796 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.576014  [16384/60000]\n",
      "loss: 0.576862  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.3%, Avg loss: 0.604275 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.575431  [16384/60000]\n",
      "loss: 0.576355  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.3%, Avg loss: 0.603733 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.574880  [16384/60000]\n",
      "loss: 0.575740  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.3%, Avg loss: 0.603245 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.574300  [16384/60000]\n",
      "loss: 0.575249  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.4%, Avg loss: 0.602697 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.573729  [16384/60000]\n",
      "loss: 0.574672  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.4%, Avg loss: 0.602230 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.573179  [16384/60000]\n",
      "loss: 0.574140  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.4%, Avg loss: 0.601668 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.572599  [16384/60000]\n",
      "loss: 0.573564  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.4%, Avg loss: 0.601184 \n",
      "\n",
      "Evolution 30\n",
      "*********************************************\n",
      "Params=350410\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.570641  [16384/60000]\n",
      "loss: 0.571714  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.4%, Avg loss: 0.599490 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.570099  [16384/60000]\n",
      "loss: 0.571155  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.4%, Avg loss: 0.598975 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.569511  [16384/60000]\n",
      "loss: 0.570584  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.4%, Avg loss: 0.598444 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.569030  [16384/60000]\n",
      "loss: 0.570047  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.4%, Avg loss: 0.597890 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.568446  [16384/60000]\n",
      "loss: 0.569575  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.5%, Avg loss: 0.597413 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.567894  [16384/60000]\n",
      "loss: 0.569056  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.5%, Avg loss: 0.596907 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.567391  [16384/60000]\n",
      "loss: 0.568556  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.5%, Avg loss: 0.596495 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.566881  [16384/60000]\n",
      "loss: 0.568044  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.5%, Avg loss: 0.595959 \n",
      "\n",
      "Evolution 31\n",
      "*********************************************\n",
      "Params=362890\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.566359  [16384/60000]\n",
      "loss: 0.567583  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.5%, Avg loss: 0.595461 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.565850  [16384/60000]\n",
      "loss: 0.567113  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.5%, Avg loss: 0.594992 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.565344  [16384/60000]\n",
      "loss: 0.566626  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.5%, Avg loss: 0.594494 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.564840  [16384/60000]\n",
      "loss: 0.566176  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.5%, Avg loss: 0.594004 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.564326  [16384/60000]\n",
      "loss: 0.565694  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.5%, Avg loss: 0.593541 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.563859  [16384/60000]\n",
      "loss: 0.565163  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.5%, Avg loss: 0.593072 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.563414  [16384/60000]\n",
      "loss: 0.564718  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.5%, Avg loss: 0.592630 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.562904  [16384/60000]\n",
      "loss: 0.564196  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.6%, Avg loss: 0.592161 \n",
      "\n",
      "Evolution 32\n",
      "*********************************************\n",
      "Params=371210\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.561114  [16384/60000]\n",
      "loss: 0.562523  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.5%, Avg loss: 0.590666 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.560682  [16384/60000]\n",
      "loss: 0.562093  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.5%, Avg loss: 0.590231 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.560206  [16384/60000]\n",
      "loss: 0.561608  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.6%, Avg loss: 0.589806 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.559684  [16384/60000]\n",
      "loss: 0.561182  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.6%, Avg loss: 0.589382 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.559242  [16384/60000]\n",
      "loss: 0.560702  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.6%, Avg loss: 0.588876 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.558820  [16384/60000]\n",
      "loss: 0.560240  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.6%, Avg loss: 0.588524 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.558398  [16384/60000]\n",
      "loss: 0.559798  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.6%, Avg loss: 0.588012 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.557903  [16384/60000]\n",
      "loss: 0.559310  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.7%, Avg loss: 0.587650 \n",
      "\n",
      "Evolution 33\n",
      "*********************************************\n",
      "Params=383690\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.557473  [16384/60000]\n",
      "loss: 0.558883  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.7%, Avg loss: 0.587175 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.557011  [16384/60000]\n",
      "loss: 0.558431  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.7%, Avg loss: 0.586761 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.556540  [16384/60000]\n",
      "loss: 0.557993  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.7%, Avg loss: 0.586352 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.556097  [16384/60000]\n",
      "loss: 0.557564  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.8%, Avg loss: 0.585922 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.555656  [16384/60000]\n",
      "loss: 0.557158  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.8%, Avg loss: 0.585569 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.555220  [16384/60000]\n",
      "loss: 0.556738  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.8%, Avg loss: 0.585097 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.554815  [16384/60000]\n",
      "loss: 0.556327  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.8%, Avg loss: 0.584723 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.554358  [16384/60000]\n",
      "loss: 0.555952  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.8%, Avg loss: 0.584241 \n",
      "\n",
      "Evolution 34\n",
      "*********************************************\n",
      "Params=392010\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.549492  [16384/60000]\n",
      "loss: 0.551428  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.7%, Avg loss: 0.580630 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.549052  [16384/60000]\n",
      "loss: 0.551057  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.7%, Avg loss: 0.579962 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.548515  [16384/60000]\n",
      "loss: 0.550581  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.7%, Avg loss: 0.579546 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.548092  [16384/60000]\n",
      "loss: 0.550263  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.8%, Avg loss: 0.579003 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.547763  [16384/60000]\n",
      "loss: 0.549861  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.8%, Avg loss: 0.578663 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.547186  [16384/60000]\n",
      "loss: 0.549372  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.8%, Avg loss: 0.578288 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.546858  [16384/60000]\n",
      "loss: 0.548948  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.8%, Avg loss: 0.577909 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.546495  [16384/60000]\n",
      "loss: 0.548462  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.8%, Avg loss: 0.577534 \n",
      "\n",
      "Evolution 35\n",
      "*********************************************\n",
      "Params=400330\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.542624  [16384/60000]\n",
      "loss: 0.545115  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.9%, Avg loss: 0.574626 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.542265  [16384/60000]\n",
      "loss: 0.544683  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.9%, Avg loss: 0.573963 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.541666  [16384/60000]\n",
      "loss: 0.544359  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.9%, Avg loss: 0.573678 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.541102  [16384/60000]\n",
      "loss: 0.543887  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.9%, Avg loss: 0.573280 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.540784  [16384/60000]\n",
      "loss: 0.543442  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.8%, Avg loss: 0.572972 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.540324  [16384/60000]\n",
      "loss: 0.543117  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.9%, Avg loss: 0.572436 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.539903  [16384/60000]\n",
      "loss: 0.542657  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.8%, Avg loss: 0.572160 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.539572  [16384/60000]\n",
      "loss: 0.542055  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.9%, Avg loss: 0.571683 \n",
      "\n",
      "Evolution 36\n",
      "*********************************************\n",
      "Params=404490\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.539041  [16384/60000]\n",
      "loss: 0.541532  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.9%, Avg loss: 0.571159 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.538539  [16384/60000]\n",
      "loss: 0.541126  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.9%, Avg loss: 0.570770 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.538078  [16384/60000]\n",
      "loss: 0.540548  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.9%, Avg loss: 0.570347 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.537601  [16384/60000]\n",
      "loss: 0.540254  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.9%, Avg loss: 0.570119 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.537191  [16384/60000]\n",
      "loss: 0.539800  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 79.9%, Avg loss: 0.569432 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.537037  [16384/60000]\n",
      "loss: 0.539557  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.0%, Avg loss: 0.568996 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.536377  [16384/60000]\n",
      "loss: 0.539038  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.0%, Avg loss: 0.568674 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.535966  [16384/60000]\n",
      "loss: 0.538848  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.1%, Avg loss: 0.568396 \n",
      "\n",
      "Evolution 37\n",
      "*********************************************\n",
      "Params=416970\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.535772  [16384/60000]\n",
      "loss: 0.538189  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.1%, Avg loss: 0.568137 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.535282  [16384/60000]\n",
      "loss: 0.537880  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.1%, Avg loss: 0.567718 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.534966  [16384/60000]\n",
      "loss: 0.537205  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.1%, Avg loss: 0.567307 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.534632  [16384/60000]\n",
      "loss: 0.537043  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.1%, Avg loss: 0.566858 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.534232  [16384/60000]\n",
      "loss: 0.536654  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.1%, Avg loss: 0.566333 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.533738  [16384/60000]\n",
      "loss: 0.536314  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.2%, Avg loss: 0.566224 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.533456  [16384/60000]\n",
      "loss: 0.535769  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.2%, Avg loss: 0.565800 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.533148  [16384/60000]\n",
      "loss: 0.535495  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.2%, Avg loss: 0.565548 \n",
      "\n",
      "Evolution 38\n",
      "*********************************************\n",
      "Params=425290\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.530013  [16384/60000]\n",
      "loss: 0.532672  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.2%, Avg loss: 0.563296 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.529783  [16384/60000]\n",
      "loss: 0.532419  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.3%, Avg loss: 0.563164 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.529386  [16384/60000]\n",
      "loss: 0.532006  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.3%, Avg loss: 0.562697 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.528566  [16384/60000]\n",
      "loss: 0.531834  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.2%, Avg loss: 0.562248 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.528294  [16384/60000]\n",
      "loss: 0.531192  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.3%, Avg loss: 0.561493 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.527925  [16384/60000]\n",
      "loss: 0.530808  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.3%, Avg loss: 0.561332 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.527590  [16384/60000]\n",
      "loss: 0.530342  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.3%, Avg loss: 0.560537 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.527411  [16384/60000]\n",
      "loss: 0.529812  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.3%, Avg loss: 0.560449 \n",
      "\n",
      "Evolution 39\n",
      "*********************************************\n",
      "Params=433610\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.526609  [16384/60000]\n",
      "loss: 0.529362  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.4%, Avg loss: 0.559821 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.526364  [16384/60000]\n",
      "loss: 0.528868  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.4%, Avg loss: 0.559383 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.525802  [16384/60000]\n",
      "loss: 0.528664  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.5%, Avg loss: 0.559231 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.525655  [16384/60000]\n",
      "loss: 0.528166  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.3%, Avg loss: 0.559073 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.525281  [16384/60000]\n",
      "loss: 0.527964  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.4%, Avg loss: 0.558848 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.524752  [16384/60000]\n",
      "loss: 0.527600  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.5%, Avg loss: 0.558231 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.524416  [16384/60000]\n",
      "loss: 0.527033  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.5%, Avg loss: 0.557930 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.524084  [16384/60000]\n",
      "loss: 0.526807  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.6%, Avg loss: 0.557407 \n",
      "\n",
      "Evolution 40\n",
      "*********************************************\n",
      "Params=446090\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.523971  [16384/60000]\n",
      "loss: 0.526346  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.4%, Avg loss: 0.557223 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.523495  [16384/60000]\n",
      "loss: 0.525870  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.6%, Avg loss: 0.556754 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.523020  [16384/60000]\n",
      "loss: 0.525611  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.5%, Avg loss: 0.556471 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.522655  [16384/60000]\n",
      "loss: 0.525349  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.6%, Avg loss: 0.556335 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.522305  [16384/60000]\n",
      "loss: 0.524892  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.7%, Avg loss: 0.556277 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.522249  [16384/60000]\n",
      "loss: 0.524637  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.7%, Avg loss: 0.555597 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.521597  [16384/60000]\n",
      "loss: 0.524296  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.6%, Avg loss: 0.555163 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.521372  [16384/60000]\n",
      "loss: 0.523621  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.6%, Avg loss: 0.554966 \n",
      "\n",
      "Evolution 41\n",
      "*********************************************\n",
      "Params=458570\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.521012  [16384/60000]\n",
      "loss: 0.523329  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.7%, Avg loss: 0.554325 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.520636  [16384/60000]\n",
      "loss: 0.523060  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.7%, Avg loss: 0.554052 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.519979  [16384/60000]\n",
      "loss: 0.522963  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.7%, Avg loss: 0.553764 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.519946  [16384/60000]\n",
      "loss: 0.522459  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.7%, Avg loss: 0.553480 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.519226  [16384/60000]\n",
      "loss: 0.522052  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.7%, Avg loss: 0.553142 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.518977  [16384/60000]\n",
      "loss: 0.522026  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.7%, Avg loss: 0.552765 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.518722  [16384/60000]\n",
      "loss: 0.521382  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.6%, Avg loss: 0.552687 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.518455  [16384/60000]\n",
      "loss: 0.521292  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.7%, Avg loss: 0.552229 \n",
      "\n",
      "Evolution 42\n",
      "*********************************************\n",
      "Params=466890\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.516694  [16384/60000]\n",
      "loss: 0.519629  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.8%, Avg loss: 0.551333 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.516342  [16384/60000]\n",
      "loss: 0.519533  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.8%, Avg loss: 0.550759 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.516205  [16384/60000]\n",
      "loss: 0.519211  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.9%, Avg loss: 0.550561 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.515995  [16384/60000]\n",
      "loss: 0.518900  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.8%, Avg loss: 0.550366 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.516000  [16384/60000]\n",
      "loss: 0.518664  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.9%, Avg loss: 0.549849 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.515096  [16384/60000]\n",
      "loss: 0.518102  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.9%, Avg loss: 0.549483 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.514638  [16384/60000]\n",
      "loss: 0.517888  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.8%, Avg loss: 0.548800 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.514567  [16384/60000]\n",
      "loss: 0.518134  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.8%, Avg loss: 0.548262 \n",
      "\n",
      "Evolution 43\n",
      "*********************************************\n",
      "Params=479370\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.514426  [16384/60000]\n",
      "loss: 0.517308  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.8%, Avg loss: 0.548457 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.514186  [16384/60000]\n",
      "loss: 0.516943  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.8%, Avg loss: 0.548062 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.513690  [16384/60000]\n",
      "loss: 0.516483  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.8%, Avg loss: 0.547529 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.513076  [16384/60000]\n",
      "loss: 0.516319  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.8%, Avg loss: 0.547556 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.512928  [16384/60000]\n",
      "loss: 0.516152  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.8%, Avg loss: 0.547392 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.512598  [16384/60000]\n",
      "loss: 0.515850  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.8%, Avg loss: 0.547164 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.512376  [16384/60000]\n",
      "loss: 0.515412  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.9%, Avg loss: 0.546730 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.511954  [16384/60000]\n",
      "loss: 0.514806  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.8%, Avg loss: 0.546489 \n",
      "\n",
      "Evolution 44\n",
      "*********************************************\n",
      "Params=496010\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.511593  [16384/60000]\n",
      "loss: 0.514517  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.8%, Avg loss: 0.546256 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.511215  [16384/60000]\n",
      "loss: 0.514366  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.9%, Avg loss: 0.545873 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.511112  [16384/60000]\n",
      "loss: 0.513906  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.8%, Avg loss: 0.545390 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.510920  [16384/60000]\n",
      "loss: 0.513859  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.8%, Avg loss: 0.545133 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.510431  [16384/60000]\n",
      "loss: 0.513531  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.9%, Avg loss: 0.544503 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.510357  [16384/60000]\n",
      "loss: 0.513290  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.0%, Avg loss: 0.544223 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.509986  [16384/60000]\n",
      "loss: 0.512979  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.0%, Avg loss: 0.544273 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.509523  [16384/60000]\n",
      "loss: 0.512616  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.9%, Avg loss: 0.543761 \n",
      "\n",
      "Evolution 45\n",
      "*********************************************\n",
      "Params=508490\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.509011  [16384/60000]\n",
      "loss: 0.512371  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.0%, Avg loss: 0.543723 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.508815  [16384/60000]\n",
      "loss: 0.511939  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.9%, Avg loss: 0.543320 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.508662  [16384/60000]\n",
      "loss: 0.511955  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.9%, Avg loss: 0.542837 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.508492  [16384/60000]\n",
      "loss: 0.511472  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.0%, Avg loss: 0.542513 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.508200  [16384/60000]\n",
      "loss: 0.511068  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.0%, Avg loss: 0.542386 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.508097  [16384/60000]\n",
      "loss: 0.510591  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.0%, Avg loss: 0.542078 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.507685  [16384/60000]\n",
      "loss: 0.510393  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.0%, Avg loss: 0.542008 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.506988  [16384/60000]\n",
      "loss: 0.510183  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 80.9%, Avg loss: 0.541617 \n",
      "\n",
      "Evolution 46\n",
      "*********************************************\n",
      "Params=520970\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.506915  [16384/60000]\n",
      "loss: 0.510026  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.0%, Avg loss: 0.541130 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.506690  [16384/60000]\n",
      "loss: 0.509773  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.1%, Avg loss: 0.540810 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.506436  [16384/60000]\n",
      "loss: 0.509405  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.0%, Avg loss: 0.540683 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.506392  [16384/60000]\n",
      "loss: 0.509153  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.0%, Avg loss: 0.540476 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.505868  [16384/60000]\n",
      "loss: 0.508813  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.0%, Avg loss: 0.540323 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.505532  [16384/60000]\n",
      "loss: 0.508589  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.2%, Avg loss: 0.540049 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.505232  [16384/60000]\n",
      "loss: 0.508223  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.2%, Avg loss: 0.539791 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.504637  [16384/60000]\n",
      "loss: 0.507950  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.0%, Avg loss: 0.539565 \n",
      "\n",
      "Evolution 47\n",
      "*********************************************\n",
      "Params=529290\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.503814  [16384/60000]\n",
      "loss: 0.506983  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.2%, Avg loss: 0.538749 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.503521  [16384/60000]\n",
      "loss: 0.506837  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.1%, Avg loss: 0.538495 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.503367  [16384/60000]\n",
      "loss: 0.506864  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.1%, Avg loss: 0.538614 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.503271  [16384/60000]\n",
      "loss: 0.506521  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.2%, Avg loss: 0.538033 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.503041  [16384/60000]\n",
      "loss: 0.506121  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.2%, Avg loss: 0.538097 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.502772  [16384/60000]\n",
      "loss: 0.505768  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.2%, Avg loss: 0.537717 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.502570  [16384/60000]\n",
      "loss: 0.505610  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.2%, Avg loss: 0.537171 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.502088  [16384/60000]\n",
      "loss: 0.505102  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.2%, Avg loss: 0.537063 \n",
      "\n",
      "Evolution 48\n",
      "*********************************************\n",
      "Params=541770\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.501764  [16384/60000]\n",
      "loss: 0.504856  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.2%, Avg loss: 0.537073 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.501721  [16384/60000]\n",
      "loss: 0.504493  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.2%, Avg loss: 0.536622 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.501556  [16384/60000]\n",
      "loss: 0.504191  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.3%, Avg loss: 0.536344 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.501191  [16384/60000]\n",
      "loss: 0.504068  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.2%, Avg loss: 0.536605 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.501218  [16384/60000]\n",
      "loss: 0.503703  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.3%, Avg loss: 0.536022 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.500584  [16384/60000]\n",
      "loss: 0.503453  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.2%, Avg loss: 0.536033 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.500587  [16384/60000]\n",
      "loss: 0.503287  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.3%, Avg loss: 0.535760 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.500437  [16384/60000]\n",
      "loss: 0.503046  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.3%, Avg loss: 0.535294 \n",
      "\n",
      "Evolution 49\n",
      "*********************************************\n",
      "Params=545930\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.499483  [16384/60000]\n",
      "loss: 0.502390  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.4%, Avg loss: 0.534763 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.499530  [16384/60000]\n",
      "loss: 0.502055  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.5%, Avg loss: 0.534508 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.498749  [16384/60000]\n",
      "loss: 0.501662  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.3%, Avg loss: 0.534115 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.498651  [16384/60000]\n",
      "loss: 0.501481  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.5%, Avg loss: 0.533919 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.497907  [16384/60000]\n",
      "loss: 0.501488  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.4%, Avg loss: 0.533898 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.498084  [16384/60000]\n",
      "loss: 0.501130  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.2%, Avg loss: 0.533653 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.497898  [16384/60000]\n",
      "loss: 0.500850  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.4%, Avg loss: 0.533398 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.497423  [16384/60000]\n",
      "loss: 0.500452  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.5%, Avg loss: 0.532908 \n",
      "\n",
      "Evolution 50\n",
      "*********************************************\n",
      "Params=558410\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.496880  [16384/60000]\n",
      "loss: 0.499972  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.6%, Avg loss: 0.532595 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.496590  [16384/60000]\n",
      "loss: 0.500052  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.5%, Avg loss: 0.532648 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.496397  [16384/60000]\n",
      "loss: 0.499788  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.5%, Avg loss: 0.532187 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.496558  [16384/60000]\n",
      "loss: 0.499451  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.4%, Avg loss: 0.531951 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.496335  [16384/60000]\n",
      "loss: 0.499093  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.4%, Avg loss: 0.531892 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.495857  [16384/60000]\n",
      "loss: 0.499067  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.5%, Avg loss: 0.531805 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.495513  [16384/60000]\n",
      "loss: 0.498822  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.5%, Avg loss: 0.531462 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.495434  [16384/60000]\n",
      "loss: 0.498584  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.5%, Avg loss: 0.531405 \n",
      "\n",
      "Evolution 51\n",
      "*********************************************\n",
      "Params=566730\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.494743  [16384/60000]\n",
      "loss: 0.497697  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.5%, Avg loss: 0.530700 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.494572  [16384/60000]\n",
      "loss: 0.497618  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.4%, Avg loss: 0.530808 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.494320  [16384/60000]\n",
      "loss: 0.496966  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.5%, Avg loss: 0.530016 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.493948  [16384/60000]\n",
      "loss: 0.497241  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.5%, Avg loss: 0.530003 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.493696  [16384/60000]\n",
      "loss: 0.496922  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.3%, Avg loss: 0.529936 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.493764  [16384/60000]\n",
      "loss: 0.496822  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.3%, Avg loss: 0.529528 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.493240  [16384/60000]\n",
      "loss: 0.496522  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.4%, Avg loss: 0.529262 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.492821  [16384/60000]\n",
      "loss: 0.496161  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.6%, Avg loss: 0.529178 \n",
      "\n",
      "Evolution 52\n",
      "*********************************************\n",
      "Params=575050\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.492203  [16384/60000]\n",
      "loss: 0.495480  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.3%, Avg loss: 0.529130 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.492085  [16384/60000]\n",
      "loss: 0.494966  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.4%, Avg loss: 0.528471 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.491368  [16384/60000]\n",
      "loss: 0.495084  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.5%, Avg loss: 0.528214 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.491082  [16384/60000]\n",
      "loss: 0.495247  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.5%, Avg loss: 0.528229 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.491034  [16384/60000]\n",
      "loss: 0.495093  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.6%, Avg loss: 0.527834 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.490532  [16384/60000]\n",
      "loss: 0.494756  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.5%, Avg loss: 0.527786 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.490239  [16384/60000]\n",
      "loss: 0.494693  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.6%, Avg loss: 0.527623 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.490514  [16384/60000]\n",
      "loss: 0.494044  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.7%, Avg loss: 0.527250 \n",
      "\n",
      "Evolution 53\n",
      "*********************************************\n",
      "Params=587530\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.489895  [16384/60000]\n",
      "loss: 0.493581  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.7%, Avg loss: 0.526998 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.489682  [16384/60000]\n",
      "loss: 0.493275  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.6%, Avg loss: 0.526709 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.489268  [16384/60000]\n",
      "loss: 0.493444  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.7%, Avg loss: 0.526694 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.489384  [16384/60000]\n",
      "loss: 0.492979  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.7%, Avg loss: 0.525968 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.489163  [16384/60000]\n",
      "loss: 0.492712  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.6%, Avg loss: 0.526081 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.488947  [16384/60000]\n",
      "loss: 0.492680  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.7%, Avg loss: 0.526266 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.488729  [16384/60000]\n",
      "loss: 0.492776  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.6%, Avg loss: 0.525937 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.488525  [16384/60000]\n",
      "loss: 0.492785  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.6%, Avg loss: 0.525239 \n",
      "\n",
      "Evolution 54\n",
      "*********************************************\n",
      "Params=595850\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.486998  [16384/60000]\n",
      "loss: 0.491016  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.6%, Avg loss: 0.524702 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.486786  [16384/60000]\n",
      "loss: 0.490956  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.6%, Avg loss: 0.524511 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.486569  [16384/60000]\n",
      "loss: 0.490933  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.5%, Avg loss: 0.524412 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.486681  [16384/60000]\n",
      "loss: 0.490957  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.6%, Avg loss: 0.524225 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.487095  [16384/60000]\n",
      "loss: 0.491063  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.6%, Avg loss: 0.524040 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.486623  [16384/60000]\n",
      "loss: 0.490545  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.6%, Avg loss: 0.523646 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.485890  [16384/60000]\n",
      "loss: 0.490226  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.7%, Avg loss: 0.523850 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.485834  [16384/60000]\n",
      "loss: 0.489948  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.7%, Avg loss: 0.523160 \n",
      "\n",
      "Evolution 55\n",
      "*********************************************\n",
      "Params=600010\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.485458  [16384/60000]\n",
      "loss: 0.489678  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.6%, Avg loss: 0.523024 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.485607  [16384/60000]\n",
      "loss: 0.489572  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.6%, Avg loss: 0.522840 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.485347  [16384/60000]\n",
      "loss: 0.489142  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.5%, Avg loss: 0.522652 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.485092  [16384/60000]\n",
      "loss: 0.488861  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.7%, Avg loss: 0.522923 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.484963  [16384/60000]\n",
      "loss: 0.488785  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.6%, Avg loss: 0.522784 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.485526  [16384/60000]\n",
      "loss: 0.489401  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.7%, Avg loss: 0.522468 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.484333  [16384/60000]\n",
      "loss: 0.488920  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.8%, Avg loss: 0.523014 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.484673  [16384/60000]\n",
      "loss: 0.488529  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.7%, Avg loss: 0.521790 \n",
      "\n",
      "Evolution 56\n",
      "*********************************************\n",
      "Params=608330\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.482117  [16384/60000]\n",
      "loss: 0.487495  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.7%, Avg loss: 0.521200 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.481954  [16384/60000]\n",
      "loss: 0.488528  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.6%, Avg loss: 0.521774 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.482407  [16384/60000]\n",
      "loss: 0.486711  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.6%, Avg loss: 0.520961 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.481368  [16384/60000]\n",
      "loss: 0.486553  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.8%, Avg loss: 0.520228 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.481824  [16384/60000]\n",
      "loss: 0.485878  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.7%, Avg loss: 0.520015 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.481032  [16384/60000]\n",
      "loss: 0.485662  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.7%, Avg loss: 0.519652 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.481306  [16384/60000]\n",
      "loss: 0.485630  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.7%, Avg loss: 0.519700 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.481373  [16384/60000]\n",
      "loss: 0.484969  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.7%, Avg loss: 0.519185 \n",
      "\n",
      "Evolution 57\n",
      "*********************************************\n",
      "Params=616650\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.480641  [16384/60000]\n",
      "loss: 0.484703  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.7%, Avg loss: 0.519103 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.480086  [16384/60000]\n",
      "loss: 0.484249  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.7%, Avg loss: 0.519032 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.479464  [16384/60000]\n",
      "loss: 0.484480  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.8%, Avg loss: 0.519032 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.479500  [16384/60000]\n",
      "loss: 0.483869  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.8%, Avg loss: 0.519100 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.479292  [16384/60000]\n",
      "loss: 0.483919  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.7%, Avg loss: 0.518665 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.479543  [16384/60000]\n",
      "loss: 0.483537  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.8%, Avg loss: 0.518071 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.479419  [16384/60000]\n",
      "loss: 0.483583  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.8%, Avg loss: 0.518260 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.479468  [16384/60000]\n",
      "loss: 0.483226  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.8%, Avg loss: 0.517630 \n",
      "\n",
      "Evolution 58\n",
      "*********************************************\n",
      "Params=629130\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.479240  [16384/60000]\n",
      "loss: 0.482715  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.8%, Avg loss: 0.517357 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.478737  [16384/60000]\n",
      "loss: 0.482371  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.7%, Avg loss: 0.516808 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.479780  [16384/60000]\n",
      "loss: 0.483717  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.8%, Avg loss: 0.517277 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.479186  [16384/60000]\n",
      "loss: 0.482514  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.7%, Avg loss: 0.516843 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.478819  [16384/60000]\n",
      "loss: 0.483125  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.8%, Avg loss: 0.517238 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.477579  [16384/60000]\n",
      "loss: 0.482415  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.7%, Avg loss: 0.517176 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.477794  [16384/60000]\n",
      "loss: 0.482068  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.8%, Avg loss: 0.516864 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.477410  [16384/60000]\n",
      "loss: 0.481995  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.8%, Avg loss: 0.517011 \n",
      "\n",
      "Evolution 59\n",
      "*********************************************\n",
      "Params=637450\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.476393  [16384/60000]\n",
      "loss: 0.482486  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.6%, Avg loss: 0.516527 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.477400  [16384/60000]\n",
      "loss: 0.481879  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.6%, Avg loss: 0.516794 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.477108  [16384/60000]\n",
      "loss: 0.482036  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.8%, Avg loss: 0.515322 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.476330  [16384/60000]\n",
      "loss: 0.480987  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 82.0%, Avg loss: 0.514980 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.475553  [16384/60000]\n",
      "loss: 0.480393  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 82.0%, Avg loss: 0.514804 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.475282  [16384/60000]\n",
      "loss: 0.480612  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 82.0%, Avg loss: 0.514675 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.475691  [16384/60000]\n",
      "loss: 0.480088  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.8%, Avg loss: 0.514324 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.475494  [16384/60000]\n",
      "loss: 0.480564  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.9%, Avg loss: 0.515527 \n",
      "\n",
      "Evolution 60\n",
      "*********************************************\n",
      "Params=654090\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.474931  [16384/60000]\n",
      "loss: 0.480943  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 82.0%, Avg loss: 0.515141 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.474216  [16384/60000]\n",
      "loss: 0.479950  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 82.0%, Avg loss: 0.514968 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.474522  [16384/60000]\n",
      "loss: 0.479291  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.9%, Avg loss: 0.513317 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.474360  [16384/60000]\n",
      "loss: 0.479092  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.9%, Avg loss: 0.514424 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.475090  [16384/60000]\n",
      "loss: 0.478981  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.8%, Avg loss: 0.513298 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.474242  [16384/60000]\n",
      "loss: 0.479522  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.8%, Avg loss: 0.513841 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.474766  [16384/60000]\n",
      "loss: 0.478461  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.8%, Avg loss: 0.513763 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.474083  [16384/60000]\n",
      "loss: 0.477876  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.7%, Avg loss: 0.513472 \n",
      "\n",
      "Evolution 61\n",
      "*********************************************\n",
      "Params=666570\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.473730  [16384/60000]\n",
      "loss: 0.477626  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.8%, Avg loss: 0.512261 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.473464  [16384/60000]\n",
      "loss: 0.481716  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.8%, Avg loss: 0.513151 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.475089  [16384/60000]\n",
      "loss: 0.478513  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.9%, Avg loss: 0.512529 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.474548  [16384/60000]\n",
      "loss: 0.478639  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 82.0%, Avg loss: 0.511827 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.473996  [16384/60000]\n",
      "loss: 0.477684  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 82.0%, Avg loss: 0.510854 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.473450  [16384/60000]\n",
      "loss: 0.477145  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 82.0%, Avg loss: 0.510624 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.472766  [16384/60000]\n",
      "loss: 0.477010  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 82.1%, Avg loss: 0.510815 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.473326  [16384/60000]\n",
      "loss: 0.476764  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 82.0%, Avg loss: 0.510858 \n",
      "\n",
      "Evolution 62\n",
      "*********************************************\n",
      "Params=674890\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.472753  [16384/60000]\n",
      "loss: 0.476154  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 82.0%, Avg loss: 0.510560 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.472846  [16384/60000]\n",
      "loss: 0.475909  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 82.0%, Avg loss: 0.511054 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.472354  [16384/60000]\n",
      "loss: 0.476033  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 82.0%, Avg loss: 0.511049 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.473010  [16384/60000]\n",
      "loss: 0.475526  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 82.0%, Avg loss: 0.509795 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.471674  [16384/60000]\n",
      "loss: 0.475204  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 82.0%, Avg loss: 0.511496 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.473153  [16384/60000]\n",
      "loss: 0.476268  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.9%, Avg loss: 0.511525 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.472554  [16384/60000]\n",
      "loss: 0.475617  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 82.0%, Avg loss: 0.511404 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.472544  [16384/60000]\n",
      "loss: 0.475179  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.9%, Avg loss: 0.510115 \n",
      "\n",
      "Evolution 63\n",
      "*********************************************\n",
      "Params=683210\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.469943  [16384/60000]\n",
      "loss: 0.475822  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.9%, Avg loss: 0.511154 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.469629  [16384/60000]\n",
      "loss: 0.476847  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.9%, Avg loss: 0.511471 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.470226  [16384/60000]\n",
      "loss: 0.473850  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 82.0%, Avg loss: 0.509776 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.469550  [16384/60000]\n",
      "loss: 0.473760  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 82.1%, Avg loss: 0.509065 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.468379  [16384/60000]\n",
      "loss: 0.473900  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.8%, Avg loss: 0.514036 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.473819  [16384/60000]\n",
      "loss: 0.474629  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.9%, Avg loss: 0.510078 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.470178  [16384/60000]\n",
      "loss: 0.474294  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.8%, Avg loss: 0.510507 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.469913  [16384/60000]\n",
      "loss: 0.473438  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.8%, Avg loss: 0.510477 \n",
      "\n",
      "Evolution 64\n",
      "*********************************************\n",
      "Params=687370\n",
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 0.470510  [16384/60000]\n",
      "loss: 0.474532  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.9%, Avg loss: 0.510851 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 0.470094  [16384/60000]\n",
      "loss: 0.473155  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 82.1%, Avg loss: 0.508611 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 0.468219  [16384/60000]\n",
      "loss: 0.472682  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.9%, Avg loss: 0.508499 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 0.468809  [16384/60000]\n",
      "loss: 0.472277  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 82.0%, Avg loss: 0.507735 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 0.468902  [16384/60000]\n",
      "loss: 0.472213  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 82.0%, Avg loss: 0.509382 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 0.469317  [16384/60000]\n",
      "loss: 0.472175  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 82.0%, Avg loss: 0.507879 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 0.468680  [16384/60000]\n",
      "loss: 0.471913  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 81.8%, Avg loss: 0.509416 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.469860  [16384/60000]\n",
      "loss: 0.473512  [49152/60000]\n",
      "Test Error: \n",
      " Accuracy: 82.0%, Avg loss: 0.509448 \n",
      "\n",
      "Done!\n"
     ]
    }
   ],
   "source": [
    "loss_fn = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=1e-3)\n",
    "\n",
    "evolutions = 64\n",
    "for e in range(evolutions):\n",
    "    c = sum(p.numel() for p in model.parameters())\n",
    "    print(f\"Evolution {e+1}\\n*********************************************\")\n",
    "    print(f\"Params={c}\")\n",
    "    epochs = 8\n",
    "    for t in range(epochs):\n",
    "        print(f\"Epoch {t+1}\\n-------------------------------\")\n",
    "        train(train_dataloader, model, loss_fn, optimizer)\n",
    "        test(test_dataloader, model, loss_fn)\n",
    "    model.split_random()\n",
    "    \n",
    "print(\"Done!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchview import draw_graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_graph = draw_graph(\n",
    "    model, \n",
    "    input_size=(1, 28*28),\n",
    "    graph_name='test',\n",
    "    graph_dir='LR',\n",
    "    depth=8,\n",
    "    hide_inner_tensors=True,\n",
    "    hide_module_functions=True,\n",
    "    save_graph=True,\n",
    "    expand_nested=True,\n",
    "    show_shapes=True,\n",
    "    filename='test',\n",
    "    directory='.'\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
